{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"HF_ENDPOINT\"] = \"https://hf-mirror.com\"\n",
    "# os.environ[\"HTTP_PROXY\"] = \"http://127.0.0.1:7890\"\n",
    "# os.environ[\"HTTPS_PROXY\"] = \"http://127.0.0.1:7890\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mindspore\n",
    "import numpy as np\n",
    "from mindnlp.evaluate import load\n",
    "from mindnlp.engine import TrainingArguments, Trainer\n",
    "from mindnlp.dataset import load_dataset\n",
    "from mindnlp.transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "from mindnlp.peft import get_peft_model, PeftModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-chinese\")\n",
    "dataset = load_dataset(\"lansinuote/ChnSentiCorp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text, label):\n",
    "    output = tokenizer(\n",
    "        text,\n",
    "        padding=\"max_length\",\n",
    "        truncation=True,\n",
    "        max_length=128,\n",
    "    )\n",
    "    return (\n",
    "        output[\"input_ids\"],\n",
    "        output[\"token_type_ids\"],\n",
    "        output[\"attention_mask\"],\n",
    "        label,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def datapipe(dataset):\n",
    "    for split in dataset.keys():\n",
    "        dataset[split] = dataset[split].map(\n",
    "            tokenize,\n",
    "            input_columns=[\"text\", \"label\"],\n",
    "            output_columns=[\"input_ids\", \"token_type_ids\", \"attention_mask\", \"labels\"],\n",
    "        )\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datapipe(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {0: \"NEGATIVE\", 1: \"POSITIVE\"}\n",
    "label2id = {\"NEGATIVE\": 0, \"POSITIVE\": 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following parameters in checkpoint files are not loaded:\n",
      "['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    \"google-bert/bert-base-chinese\",\n",
    "    num_labels=2,\n",
    "    id2label=id2label,\n",
    "    label2id=label2id,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mindnlp.peft import LoHaConfig, TaskType\n",
    "\n",
    "peft_config = LoHaConfig(\n",
    "    task_type=TaskType.SEQ_CLS,\n",
    "    inference_mode=False,\n",
    "    r=8,\n",
    "    alpha=32,\n",
    "    target_cells=[\"query\"],\n",
    "    modules_to_save=[\"classifier\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_peft_model(model, peft_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = dataset[\"train\"]\n",
    "test_loader = dataset[\"test\"]\n",
    "val_loader = dataset[\"validation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_metric = load(\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training and evaluation\n",
    "def compute_metrics(eval_preds):\n",
    "    preds, labels = eval_preds\n",
    "    preds = np.argmax(preds, axis=1)\n",
    "    accuracy = accuracy_metric.compute(predictions=preds, references=labels)\n",
    "    return {\"accuracy\": accuracy}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    \"output\",\n",
    "    per_device_train_batch_size=64,\n",
    "    per_device_eval_batch_size=64,\n",
    "    learning_rate=1e-4,\n",
    "    num_train_epochs=10,\n",
    "    evaluation_strategy=\"epoch\",\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_args,\n",
    "    train_dataset=train_loader,\n",
    "    eval_dataset=test_loader,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/1500 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 150/1500 [00:46<06:46,  3.32it/s]\n",
      " 10%|█         | 150/1500 [00:47<06:46,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5453808307647705, 'eval_accuracy': {'accuracy': 0.7475}, 'eval_runtime': 1.5895, 'eval_samples_per_second': 11.953, 'eval_steps_per_second': 0.629, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 20%|██        | 300/1500 [01:35<06:00,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.40547648072242737, 'eval_accuracy': {'accuracy': 0.8375}, 'eval_runtime': 1.4016, 'eval_samples_per_second': 13.556, 'eval_steps_per_second': 0.713, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 30%|███       | 450/1500 [02:23<05:13,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3049595057964325, 'eval_accuracy': {'accuracy': 0.8858333333333334}, 'eval_runtime': 1.7914, 'eval_samples_per_second': 10.606, 'eval_steps_per_second': 0.558, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 500/1500 [02:38<05:08,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4702, 'learning_rate': 9.999999747378752e-05, 'epoch': 3.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 40%|████      | 600/1500 [03:11<04:31,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2816271483898163, 'eval_accuracy': {'accuracy': 0.8916666666666667}, 'eval_runtime': 1.6753, 'eval_samples_per_second': 11.341, 'eval_steps_per_second': 0.597, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 50%|█████     | 750/1500 [03:59<03:43,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2704663574695587, 'eval_accuracy': {'accuracy': 0.8975}, 'eval_runtime': 1.9574, 'eval_samples_per_second': 9.707, 'eval_steps_per_second': 0.511, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 60%|██████    | 900/1500 [04:46<03:02,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2568877041339874, 'eval_accuracy': {'accuracy': 0.9033333333333333}, 'eval_runtime': 1.3997, 'eval_samples_per_second': 13.574, 'eval_steps_per_second': 0.714, 'epoch': 6.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 1000/1500 [05:17<02:35,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2686, 'learning_rate': 9.999999747378752e-05, 'epoch': 6.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 70%|███████   | 1050/1500 [05:35<02:15,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2526182234287262, 'eval_accuracy': {'accuracy': 0.9058333333333334}, 'eval_runtime': 1.8815, 'eval_samples_per_second': 10.099, 'eval_steps_per_second': 0.532, 'epoch': 7.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 80%|████████  | 1200/1500 [06:22<01:29,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.24465835094451904, 'eval_accuracy': {'accuracy': 0.9075}, 'eval_runtime': 1.396, 'eval_samples_per_second': 13.61, 'eval_steps_per_second': 0.716, 'epoch': 8.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 90%|█████████ | 1350/1500 [07:10<00:45,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.23736287653446198, 'eval_accuracy': {'accuracy': 0.9108333333333334}, 'eval_runtime': 1.9241, 'eval_samples_per_second': 9.875, 'eval_steps_per_second': 0.52, 'epoch': 9.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500/1500 [07:56<00:00,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2402, 'learning_rate': 9.999999747378752e-05, 'epoch': 10.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      "100%|██████████| 1500/1500 [07:59<00:00,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.23658284544944763, 'eval_accuracy': {'accuracy': 0.9116666666666666}, 'eval_runtime': 1.8105, 'eval_samples_per_second': 10.495, 'eval_steps_per_second': 0.552, 'epoch': 10.0}\n",
      "{'train_runtime': 479.0418, 'train_samples_per_second': 200.4, 'train_steps_per_second': 3.131, 'train_loss': 0.3263415781656901, 'epoch': 10.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1500, training_loss=0.3263415781656901, metrics={'train_runtime': 479.0418, 'train_samples_per_second': 200.4, 'train_steps_per_second': 3.131, 'train_loss': 0.3263415781656901, 'epoch': 10.0})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:01<00:00, 11.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.23658284544944763,\n",
       " 'eval_accuracy': {'accuracy': 0.9116666666666666},\n",
       " 'eval_runtime': 1.9645,\n",
       " 'eval_samples_per_second': 9.672,\n",
       " 'eval_steps_per_second': 0.509,\n",
       " 'epoch': 10.0}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"google-bert/bert-base-chinese_LoHa_SQE_CLS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ckpt = f\"google-bert/bert-base-chinese_LoHa_SQE_CLS/adapter_model.ckpt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2M\tgoogle-bert/bert-base-chinese_LoHa_SQE_CLS/adapter_model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!du -h $ckpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following parameters in checkpoint files are not loaded:\n",
      "['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.437.747 [mindspore/train/serialization.py:1378] For 'load_param_into_net', 201 parameters in the 'net' are not loaded, because they are not in the 'parameter_dict', please check whether the network structure is consistent when training and loading checkpoint.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.061 [mindspore/train/serialization.py:1383] base_model.model.bert.embeddings.word_embeddings.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.218 [mindspore/train/serialization.py:1383] base_model.model.bert.embeddings.position_embeddings.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.337 [mindspore/train/serialization.py:1383] base_model.model.bert.embeddings.token_type_embeddings.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.451 [mindspore/train/serialization.py:1383] base_model.model.bert.embeddings.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.566 [mindspore/train/serialization.py:1383] base_model.model.bert.embeddings.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.679 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.788 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.438.894 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.006 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.115 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.218 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.325 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.435 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.536 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.646 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.748 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.855 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.439.954 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.060 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.162 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.267 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.0.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.372 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.474 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.578 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.680 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.786 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.895 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.440.997 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.110 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.215 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.314 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.416 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.520 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.624 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.726 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.826 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.441.927 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.1.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.030 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.137 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.354 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.479 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.586 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.694 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.795 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.442.899 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.000 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.107 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.213 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.317 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.425 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.531 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.631 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.736 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.2.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.842 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.443.948 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.047 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.153 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.257 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.362 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.463 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.565 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.669 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.772 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.874 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.444.972 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.080 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.185 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.287 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.390 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.3.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.497 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.603 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.703 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.809 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.445.913 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.019 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.128 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.233 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.336 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.439 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.542 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.648 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.757 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.855 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.446.961 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.066 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.4.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.172 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.274 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.384 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.485 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.592 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.696 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.800 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.447.911 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.010 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.120 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.225 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.329 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.429 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.533 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.644 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.747 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.5.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.853 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.448.958 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.062 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.173 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.278 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.383 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.487 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.592 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.694 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.797 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.449.900 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.013 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.116 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.214 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.320 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.425 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.6.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.526 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.628 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.732 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.837 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.450.937 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.041 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.147 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.245 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.346 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.450 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.555 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.653 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.755 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.857 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.451.957 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.061 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.7.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.168 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.273 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.375 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.480 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.581 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.685 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.791 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.895 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.452.998 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.110 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.213 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.318 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.422 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.523 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.626 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.729 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.8.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.836 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.453.938 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.041 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.145 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.246 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.380 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.490 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.594 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.700 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.804 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.454.908 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.010 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.117 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.217 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.324 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.421 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.9.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.528 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.628 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.732 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.837 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.455.941 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.043 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.145 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.252 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.350 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.453 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.554 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.659 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.761 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.867 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.456.970 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.076 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.10.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.182 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.query.base_layer.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.292 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.query.base_layer.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.392 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.key.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.497 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.key.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.603 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.value.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.704 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.self.value.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.809 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.457.907 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.011 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.112 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.attention.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.218 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.intermediate.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.322 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.intermediate.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.425 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.output.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.533 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.output.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.645 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.output.LayerNorm.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.756 [mindspore/train/serialization.py:1383] base_model.model.bert.encoder.layer.11.output.LayerNorm.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.865 [mindspore/train/serialization.py:1383] base_model.model.bert.pooler.dense.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.458.970 [mindspore/train/serialization.py:1383] base_model.model.bert.pooler.dense.bias is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.459.081 [mindspore/train/serialization.py:1383] base_model.model.classifier.original_cell.weight is not loaded.\n",
      "[WARNING] ME(395178:138011495900992,MainProcess):2024-05-26-22:59:38.459.185 [mindspore/train/serialization.py:1383] base_model.model.classifier.original_cell.bias is not loaded.\n"
     ]
    }
   ],
   "source": [
    "new_model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    \"google-bert/bert-base-chinese\"\n",
    ")\n",
    "new_model = PeftModel.from_pretrained(\n",
    "    new_model, \"google-bert/bert-base-chinese_LoHa_SQE_CLS\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"这个服务态度真心很差劲\"\n",
    "text2 = \"他们的服务态度真的好棒\"\n",
    "output1 = tokenizer(\n",
    "    text1, return_tensors=\"ms\", truncation=True, padding=\"max_length\", max_length=128\n",
    ")\n",
    "output2 = tokenizer(\n",
    "    text2, return_tensors=\"ms\", truncation=True, padding=\"max_length\", max_length=128\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(**output1).logits.argmax().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(**output2).logits.argmax().item()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.19 ('mindspore': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2ac7b724276726859678febef8183a15210937ad3bbdc13f079040af749bbc1c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
